/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */

package net.mohas.docsearch.model;

import java.io.File;
import java.io.Reader;

import net.mohas.docsearch.Constants;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.PorterStemFilter;
import org.apache.lucene.analysis.StopAnalyzer;
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.Version;

/**
 * 
 * Class untuk melakukan text operation (operasi teks) dalam bahasa Inggris
 * dimulai dari tokenization, lowercase huruf, stop words removal, dan stemming
 * menggunakan pustaka Lucene
 * 
 * @author Muhammad, 14 April 2014
 * 
 */
public final class CustomEnglishAnalyzer extends Analyzer {

	@SuppressWarnings("resource")
	@Override
	public TokenStream tokenStream(String string, Reader reader) {

		File stopwordsFile = new File(Constants.RESOURCES_DIR
				+ Constants.STOPWORDS_DIR
				+ Constants.ENG_STOPWORDS_FILENAME);
		StopAnalyzer englishStopwords = null;
		try {
			englishStopwords = new StopAnalyzer(Version.LUCENE_33, stopwordsFile);
		} catch (Exception ex) {
			ex.printStackTrace();
		}

		// Tokenisasi menggunakan StandardTokenization
		StandardTokenizer stdTokenizer = new StandardTokenizer( Version.LUCENE_33, reader);

		// Lowercase huruf
		LowerCaseFilter lcFilter = new LowerCaseFilter(Version.LUCENE_33, stdTokenizer);

		// Stop words removal
		StopFilter stopFilter = new StopFilter(Version.LUCENE_33, lcFilter, englishStopwords.getStopwordSet(), false);
		stopFilter.setEnablePositionIncrements(true);

		// Stemming (Porter stemmer)
		PorterStemFilter stemmer = new PorterStemFilter(stopFilter);
		return stemmer;
	}

}
